package edu.brandeis.cs.steele.spider;

import java.io.BufferedReader;
import java.io.File;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.net.HttpURLConnection;
import java.net.MalformedURLException;
import java.net.URL;
import java.net.URLConnection;
import java.util.ArrayList;
import java.util.List;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

import org.apache.log4j.Logger;
import org.apache.log4j.PropertyConfigurator;

import edu.brandeis.cs.steele.util.GetAllWords;
import edu.brandeis.cs.steele.wn.POS;

public class WebSpider {
    static Logger logger = Logger.getLogger(WebSpider.class.getName());

    public static void main(String[] args) throws Exception {

        String path = "/log4j/log4j.properties";
        File f = new File("." + path);
        if (f.canWrite())
            System.out.println("can read");
        PropertyConfigurator.configure(f.getAbsolutePath());
        List<String> words = GetAllWords.getAllWords(POS.NOUN);
        // List<String> words = new ArrayList<String>();
        // /words.add("apple_maggot");
        // words.add("apple_geranium");
        // words.add("appetite");//gittern
        int i = 0;
        System.out.println("!!!!!!! =============      ========   all - "
                + words.size());
        String pattern = "[a-zA-Z0-9]+";
        // String pattern = "[\\w\\-]+";
        /*I missed following files*/
        //hamamelidaceae - 49955
        // horridness - 52937

        for (String w : words) {
            i++;
            if (i >= 50713 && i <= 52937)
                if (w.matches(pattern)) {
                    logger.debug(w + " - " + i);
                    //if(w.equals("gittern"))
                    //    return;
                    spider(w);
                    Thread.sleep(5000);
                }
        }

    }

    public static boolean IsThisWord(String word) {
        String patStr = "[\\w]+";
        Pattern pattern = Pattern.compile(patStr);
        Matcher matcher = pattern.matcher(word);
        if (matcher.find())
            return true;
        return false;
    }

    public static void spider(String word) throws Exception {
        URL url = makeDictionaryURL(word);
        if (url == null)
            return;
        HttpURLConnection connWeb = getConnectionToWebPage(url);
        String mp3UrlStr = grepWebPage(connWeb);
        URL urlMP3 = null;
        if (mp3UrlStr.isEmpty())
            return;
        try {
            urlMP3 = new URL(mp3UrlStr);
        } catch (MalformedURLException mue) {
            throw new Exception(mp3UrlStr, mue);
        }
        HttpURLConnection connMp3 = getConnectionToWebPage(urlMP3);
        saveMP3ToDisk(connMp3, word);
    }

    private static void saveMP3ToDisk(HttpURLConnection connMp3, String word)
            throws IOException {

        int contentLength = connMp3.getContentLength();
        // System.out.println(contentLength);
        // System.out.println(connMp3.getContentType());

        InputStream stream = connMp3.getInputStream();

        /*
         * while (bytesread >= 0) { bytesread = stream.read(buffer, offset,
         * bufferLength); if (bytesread == -1) break; offset += bytesread; } if
         * (offset != contentLength) { System.err.println("Error: Only read " +
         * offset + " bytes"); System.err.println("Expected " + contentLength +
         * " bytes"); }
         */
        final int buflen = Math.max(1024, Math.max(contentLength, stream
                .available()));
        byte[] buf = new byte[buflen];
        byte[] bytes = null;

        for (int nRead = stream.read(buf); nRead != -1; nRead = stream
                .read(buf)) {
            if (bytes == null) {
                bytes = buf;
                buf = new byte[buflen];
                continue;
            }
            final byte[] newBytes = new byte[bytes.length + nRead];
            System.arraycopy(bytes, 0, newBytes, 0, bytes.length);
            System.arraycopy(buf, 0, newBytes, bytes.length, nRead);
            bytes = newBytes;
        }

        StringBuilder path = new StringBuilder("c:\\english\\MP3Dictionary\\noun\\");
        path.append(word);
        path.append(".mp3");
        File f = new File(path.toString());
        // System.out.println(f.getPath());
        FileOutputStream fout = new FileOutputStream(f);
        fout.write(bytes);
        fout.close();
    }

    public static String grepURLMP3(String source) {
        // String patStr = "http://img.tfd.com/pron/mp3/en/US/st/[\\w]+\\.mp3";
        String patStr = "en/US/[\\w]{2}/[\\w]+";
        Pattern pattern = Pattern.compile(patStr);
        Matcher matcher = pattern.matcher(source);
        StringBuilder url = new StringBuilder("http://img.tfd.com/pron/mp3/");
        if (matcher.find()) {
            url.append(matcher.group());
            url.append(".mp3");
            return url.toString();
        }
        return "";
    }

    public static HttpURLConnection getConnectionToWebPage(URL url)
            throws IOException {
        java.net.HttpURLConnection conn = null;
        URLConnection uconn = url.openConnection();
        if (!(uconn instanceof java.net.HttpURLConnection))
            throw new java.lang.IllegalArgumentException(
                    "URL protocol must be HTTP.");
        conn = (HttpURLConnection) uconn;
        conn.setConnectTimeout(10000); // 10 sec
        conn.setReadTimeout(10000); // 10 sec
        conn.setInstanceFollowRedirects(true);
        conn.setRequestProperty("User-agent",
                "Mozilla/4.0 (compatible; MSIE 5.5; Windows NT 5.0)");
        return conn;
    }

    public static URL makeDictionaryURL(String word) {
        if (word.equals(""))
            return null;
        StringBuilder baseUrl = new StringBuilder(
                "http://www.thefreedictionary.com/");
        baseUrl.append(word);
        URL url = null;
        try {
            url = new URL(baseUrl.toString());
            // System.out.println(url);
        } catch (MalformedURLException e) {
            e.printStackTrace();
        }
        return url;
    }

    static public String grepWebPage(HttpURLConnection conn) throws IOException {
        InputStream str = null;
        try {
            str = conn.getInputStream();
        } catch (IOException e) {
            return "";
        }
        if (str == null)
            return "";
        BufferedReader rd = new BufferedReader(new InputStreamReader(str));
        String line;
        String res = "";
        while ((line = rd.readLine()) != null) {
            res = WebSpider.grepURLMP3(line);
            if (!res.equals(""))
                break;
        }
        conn.getInputStream().close();
        rd.close();
        return res;
    }
}
